import json
import re
import time
import redis
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
import requests
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from lg.models import Job, Company
from lxml import etree


class LagouSpider:

    def __init__(self):
        # 初始化chrome浏览器无头模式
        chrome_options = Options()
        chrome_options.add_argument('--headless')
        chrome_options.add_argument('--disable-gpu')
        self.driver = webdriver.Chrome(chrome_options=chrome_options)
        # 初始化redis链接
        self.redis_db = redis.StrictRedis(host='localhost', port=6379, decode_responses=True)
        # 初始化数据库连接
        mysql_uri = 'mysql+mysqlconnector://root:@127.0.0.1:3306/spider_data'
        engine = create_engine(mysql_uri)
        DBSession = sessionmaker(bind=engine)
        self.session = DBSession()
        # 非数字正则
        self.num_pattern = re.compile(r'\D')
        self.content_pattern = re.compile(r'\n| |\r|\t|\'|\xa0|')
        # headers
        self.headers = {
            'Accept': 'application/json, text/javascript, */*; q=0.01',
            'Accept-Encoding': 'gzip, deflate, br',
            'Accept-Language': 'zh-CN,zh;q=0.8',
            'Cache-Control': 'no-cache',
            'Connection': 'keep-alive',
            'Content-Length': '26',
            'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
            'Cookie': 'user_trace_token=20180610140405-14ae3abd-6c74-11e8-9446-5254005c3644; LGUID=20180610140405-14ae3d92-6c74-11e8-9446-5254005c3644; LG_LOGIN_USER_ID=14a21a9c55750ab158d301a1f5626d7e72fd295ae1d61e289d22e01cdb2490d9; login=true; unick=%E6%8B%89%E5%8B%BE%E7%94%A8%E6%88%B76395; showExpriedIndex=1; showExpriedCompanyHome=1; showExpriedMyPublish=1; hasDeliver=0; gate_login_token=f8b4a5aa731754b0e574b09e85c65f679b00e8beb7146c19671e73bab4f90cae; JSESSIONID=ABAAABAAAGGABCBB276D33E64CB5399407F44B3A48BE04A; PRE_UTM=; PRE_HOST=; PRE_SITE=; PRE_LAND=https%3A%2F%2Fwww.lagou.com%2Fjobs%2Flist_Python%3Fcity%3D%25E4%25B8%258A%25E6%25B5%25B7; SEARCH_ID=146e34781b9c43fa8641375beaa768a2; Hm_lvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1528610646,1528613472,1528632228; Hm_lpvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1528645233; _gid=GA1.2.914314435.1528610645; _ga=GA1.2.1432645959.1528610645; LGSID=20180610233619-05312052-6cc4-11e8-997e-525400f775ce; LGRID=20180610234032-9c09084d-6cc4-11e8-997e-525400f775ce; _putrc=3CF8638BFD7ADC62123F89F2B170EADC; index_location_city=%E5%85%A8%E5%9B%BD; TG-TRACK-CODE=search_code',
            'Host': 'www.lagou.com',
            'Origin': 'https://www.lagou.com',
            'Pragma': 'no-cache',
            'Referer': 'https://www.lagou.com/jobs/list_Python?city=%E4%B8%8A%E6%B5%B7',
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 SE 2.X MetaSr 1.0',
            'X-Anit-Forge-Code': '0',
            'X-Anit-Forge-Token': None,
            'X-Requested-With': 'XMLHttpRequest'
        }

    def get_job_company(self):
        fields = ["companyId", "companyShortName", "financeStage", "district", "companySize", "positionLables",
                  "industryField", "positionId", "positionName", "createTime", "positionAdvantage", "salary",
                  "workYear", "education", "city", "companyFullName"]
        url = 'https://www.lagou.com/jobs/positionAjax.json?city=%E4%B8%8A%E6%B5%B7&needAddtionalResult=false'
        for i in range(19):
            res = requests.post(url, data={'first': 'false', 'pn': str(i + 1), 'kd': 'Python'},
                                headers=self.headers).content.decode()
            res = json.loads(res)
            print(res)
            for job in res['content']['positionResult']['result']:
                item = {k: v for k, v in job.items() if k in fields}
                self.redis_db.lpush('lg_job_list', json.dumps(item))
            # print(i + 1)
            time.sleep(5)

    def stroe_job_company(self):
        while True:
            job_json = self.redis_db.rpop('lg_job_list')
            if job_json is None:
                print('over')
                break
            job_dict = json.loads(job_json)
            """
            {
                'companyId': 248010,
                'companyShortName': '钱包生活',
                'companySize': '2000人以上',
                'industryField': 'O2O,移动互联网',
                'financeStage': 'B轮', 
                'companyFullName': '钱包生活（平潭）科技有限公司上海分公司',
                 
                'positionName': 'Python开发工程师-运维平台方向',
                'workYear': '3-5年',
                'education': '本科',
                'salary': '20k-30k',
                'city': '上海',                      
                'positionId': 4674860,
                'createTime': '2018-06-08 17:20:28',
                'positionAdvantage': '前沿技术,极客精神,大平台,高挑战',            
                'district': '闵行区',
                'positionLables': ['linux', 'MySQL'],                     
                'companyId': 248010,
            }
            """
            salary = job_dict['salary'].split('-')
            if len(salary) == 2:
                salary_low = salary[0][:-1]
                salary_high = salary[1][:-1]
            else:
                salary_low = int(self.num_pattern.sub('', job_dict['salary']))

            # 职位去重
            if self.redis_db.sadd('lg_job_ids', job_dict['positionId']) != 0:
                # 保存到数据库
                job = Job(
                    name=job_dict['positionName'],
                    experience=job_dict['workYear'],
                    education=job_dict['education'],
                    salary_low=int(salary_low),
                    salary_high=int(salary_high),
                    city=job_dict['city'],
                    area=job_dict['district'],
                    shortcut=job_dict['positionAdvantage'],
                    company=int(job_dict['companyId']),
                    publish_time=job_dict['createTime'],
                    lagou_id=job_dict['positionId'],
                    tags=';'.join(job_dict['positionLables'])
                )
                print('职位：'+job.name)
                self.session.add(job)
                self.session.commit()
            #  公司去重
            if self.redis_db.sadd('lg_company_ids', job_dict['companyId']) != 0:
                company = Company(
                    lagou_id=int(job_dict['companyId']),
                    name=job_dict['companyShortName'],
                    size=job_dict['companySize'],
                    type=job_dict['industryField'],
                    finance=job_dict['financeStage'],
                    total_name=job_dict['companyFullName'],
                )
                print('公司：'+company.name)
                self.session.add(company)
                self.session.commit()

    def get_job_info(self):
        # 初始化chrome浏览器无头模式
        chrome_options = Options()
        chrome_options.add_argument('--headless')
        chrome_options.add_argument('--disable-gpu')
        while True:
            job_id = self.redis_db.spop('lg_job_ids')
            if job_id is None:
                print('over')
                break
            driver = webdriver.Chrome(chrome_options=chrome_options)
            job_url = 'https://www.lagou.com/jobs/{}.html'.format(job_id)
            driver.get(job_url)
            time.sleep(2)
            element = etree.HTML(driver.page_source)
            desc = element.xpath('//dd[@class="job_bt"]/div//text()')
            desc_str = ''
            for item in desc:
                desc_str += self.content_pattern.sub('', item) + '*_*'
            address = element.xpath('//div[@class="work_addr"]//text()')
            address_str = ''
            for item in address:
                address_str += self.content_pattern.sub('', item)
            driver.quit()
            # 数据过滤
            if address_str == '' or desc_str == '':
                print({'msg': '失败', 'url': job_url, 'address': address_str[:-4], 'description': desc_str})
                # self.redis_db.sadd('lg_job_ids', job_id)
                continue
            # 保存数据库
            self.session.query(Job).filter(Job.lagou_id == job_id).\
                update({'url': job_url, 'address': address_str[:-4], 'description': desc_str})
            self.session.commit()
            print({'url': job_url, 'address': address_str[:-4], 'description': desc_str})
            # time.sleep(1)

    def get_company_info(self):
        # self.session.query(Company).filter(Company.description == '').all()
        for cp in self.session.query(Company).filter(Company.description == '').all():
            self.redis_db.sadd('lg_company_ids', cp.lagou_id)
        chrome_options = Options()
        chrome_options.add_argument('--headless')
        chrome_options.add_argument('--disable-gpu')
        while True:
            company_id = self.redis_db.spop('lg_company_ids')
            if company_id is None:
                print('over')
                break
            url = 'https://www.lagou.com/gongsi/{}.html'.format(company_id)
            driver = webdriver.Chrome(chrome_options=chrome_options)
            driver.get(url)
            time.sleep(2)
            try:
                driver.find_element_by_xpath('//span[@class="text_over"]').click()
            except Exception as e:
                print(e)
            element = etree.HTML(driver.page_source)
            driver.quit()
            desc = element.xpath('//div[@class="company_intro_text"]//text()')
            imgs = ';'.join(element.xpath('//ul[@id="rotateImages"]/li/img/@src'))
            desc_str = ''
            for item in desc:
                desc_str += self.content_pattern.sub('', item) + '*_*'
            if desc_str == '':
                #self.redis_db.sadd('lg_company_ids', company_id)
                print({'msg': '失败', 'url': url, 'environments': imgs, 'description': desc_str})
                continue
            # 保存数据库
            self.session.query(Company).filter(Company.lagou_id == company_id). \
                update({'url': url, 'environments': imgs, 'description': desc_str})
            self.session.commit()
            print({'url': url, 'environments': imgs, 'description': desc_str})

    def create_wc_img(self):
        import jieba
        import matplotlib as mpl
        mpl.use("Agg")
        import matplotlib.pyplot as plt  # 数学绘图库
        import jieba  # 分词库
        from wordcloud import WordCloud  # 词云库

        jobs_info = lg.session.query(Job.tags, Job.description).all()
        description = ''
        tags = list()
        for item in jobs_info:
            description += item[1].replace('*_*', ';') + ';'
            if item[0] != '':
                tags.extend(item[0].split(';'))
        # 停用词
        no_name = ["岗位", "职责", '要求', '或者', '工作', '描述']
        desc_result = "/".join(jieba.cut(description))  # 只有用"/"或" "等分割开的，才能用于构建词云图
        # wordcloud配置
        wc = WordCloud(
            font_path="C:\Windows\Fonts\msyhbd.ttc",  # 设置字体
            background_color='rgb(236, 240, 245)',  # 背景颜色
            width=800, height=600,  # 设置宽高
            max_font_size=70, min_font_size=10,  # 字体最大/最小值
            stopwords=no_name,  # 设置停用词，不在词云图中显示
            max_words=2000,  # 设置最多显示的字数
            mode='RGBA'
        )
        # 生成职位描述信息词云图
        wc.generate(desc_result)
        img_name = 'desc_wc.png'  # 保存图片的名字
        wc.to_file(img_name)  # 生成图片
        plt.figure('词云图')  # 指定所绘图名称
        plt.imshow(wc)  # 以图片的形式显示词云
        plt.axis('off')  # 关闭图像坐标系
        plt.show()
        # 生成tag词云图
        wc.generate('/'.join(tags))
        img_name = 'tag_wc.png'
        wc.to_file(img_name)  # 生成图片
        plt.figure('词云图')  # 指定所绘图名称
        plt.imshow(wc)  # 以图片的形式显示词云
        plt.axis('off')  # 关闭图像坐标系
        plt.show()

if __name__ == '__main__':
    lg = LagouSpider()
    # lg.get_job_company()
    # lg.stroe_job_company()
    # lg.redis_db.sadd('lg_job_ids',4049446)
    # lg.redis_db.sadd('lg_job_ids',4254613)
    # lg.redis_db.sadd('lg_job_ids',4229524)
    # lg.redis_db.sadd('lg_job_ids',4254612)
    # lg.get_job_info()
    # job = lg.session.query(Job).filter_by(lagou_id=470499833333333).first()
    # print(job)
    # r1 = lg.session.execute('SELECT * FROM `lagou_job` WHERE address is NULL or description is NULL')
    # r1 = lg.session.query(Job).filter(Job.address is 'Null').all()
    # print(r1)
    # for j in r1:
    #     lg.redis_db.sadd('lg_job_ids', j.lagou_id)
    # lg.redis_db.sadd('lg_job_ids',4049446)
    # lg.redis_db.sadd('lg_job_ids',4254613)
    # lg.redis_db.sadd('lg_job_ids',4229524)
    # lg.redis_db.sadd('lg_job_ids',4254612)
    # lg.get_company_info()




